{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-14T06:51:30.121740Z",
     "start_time": "2024-11-14T06:51:27.571437Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#引用所有需要的库\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import random\n",
    "from torch.utils.checkpoint import checkpoint_sequential\n",
    "from win32gui import FlashWindowEx"
   ],
   "id": "4d518ecd2bf02391",
   "outputs": [],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-11-14T06:51:30.127853Z",
     "start_time": "2024-11-14T06:51:30.125418Z"
    }
   },
   "source": [
    "#定义贪吃蛇的动作\n",
    "snack_actions = {\n",
    "    'L': np.array([-1, 0]),\n",
    "    'R': np.array([1, 0]),\n",
    "    'U': np.array([0, -1]),\n",
    "    'D': np.array([0, 1])\n",
    "}"
   ],
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-14T06:51:30.140581Z",
     "start_time": "2024-11-14T06:51:30.136919Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#定义蛇类\n",
    "class snackclass(object):\n",
    "    def __init__(self, map_size, snack_len=4):\n",
    "        #初始化蛇的位置、运动方向、长度\n",
    "        self.map_size = map_size\n",
    "        self.position = np.array([map_size//2, map_size//2])\n",
    "        self.snack_len = snack_len\n",
    "        self.act = np.array([1, 0])\n",
    "        #列表存储蛇的历史位置\n",
    "        self.history_p = [np.array([map_size//2, map_size//2]).astype('float')]\n",
    "    #运动函数\n",
    "    def move(self):\n",
    "        self.position += self.act\n",
    "        self.history_p.append(self.position.copy())\n",
    "        self.history_p = self.history_p[-self.snack_len - 1 :]\n",
    "    #判断死亡条件：吃到自己或者碰到边界\n",
    "    def deadcheck(self,pos):\n",
    "        if pos[0] <= -1 or pos[0] >= self.map_size:\n",
    "            return True\n",
    "        elif pos[1] <= -1 or pos[1] >= self.map_size:\n",
    "            return True\n",
    "        return False\n",
    "    #获取蛇运动的合法方向\n",
    "    def getlegalaction(self):\n",
    "        L = self.position - snack_actions['L']\n",
    "        R = self.position + snack_actions['R']\n",
    "        U = self.position + snack_actions['U']\n",
    "        D = self.position + snack_actions['D']\n",
    "        checkpos = [L, R, U, D]\n",
    "        legacy = [int(self.deadcheck(p)) for p in checkpos]\n",
    "        return legacy\n",
    "    #增长\n",
    "    def __len__(self):\n",
    "        return self.snack_len + 1"
   ],
   "id": "7c2780c795b6ae0a",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-14T06:51:30.147211Z",
     "start_time": "2024-11-14T06:51:30.143945Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#定义苹果类\n",
    "class appleclass(object):\n",
    "    def __init__(self, map_size):\n",
    "        self.map_size = map_size\n",
    "        self.position = np.random.randint(1, self.map_size,2)\n",
    "        self.score = 0\n",
    "    #分数累加，刷新位置\n",
    "    def eaten(self):\n",
    "        self.position = np.random.randint(1, self.map_size,2)\n",
    "        self.score += 1"
   ],
   "id": "e56ac576b1a9452d",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-14T06:51:30.155189Z",
     "start_time": "2024-11-14T06:51:30.150285Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#定义游戏环境\n",
    "'''\n",
    "每吃一个苹果，蛇的长度和得分就会加一\n",
    "撞墙或者吃到自己，得分减一\n",
    "长时间不吃苹果，得分减一\n",
    "'''\n",
    "class environment(object):\n",
    "    def __init__(self, map_size, nothing , dead, apple):\n",
    "        self.map_size = map_size\n",
    "        self.nothing = nothing\n",
    "        self.dead = dead\n",
    "        self.apple = appleclass(map_size)\n",
    "        self.snack = snackclass(map_size)\n",
    "        self.gameover = False\n",
    "        self.applereward = apple\n",
    "        self.time_after_last = 0\n",
    "    #重置游戏\n",
    "    def reset(self):\n",
    "        self.apple.position = np.random.randint(1, self.map_size,2)\n",
    "        self.apple.score = 0\n",
    "        self.snack.position = np.random.randint(1, self.map_size,2)\n",
    "        self.snack.history_p = [self.apple.position.copy()]\n",
    "        self.snack.snack_len = 4\n",
    "        self.gameover = False\n",
    "    #获取当前游戏状态\n",
    "    def getstate(self):\n",
    "        return [self.snack.position,self.snack.act,self.snack.history_p,\n",
    "                self.apple.position,self.apple.score,self.gameover]\n",
    "    #更新游戏状态\n",
    "    def update(self,act):\n",
    "        reward = self.nothing\n",
    "        Done = False\n",
    "        if act ==0 :\n",
    "            if not (self.snack.act == snack_actions['R']).all():\n",
    "                self.snack.act = snack_actions['R']\n",
    "        if act ==1 :\n",
    "            if not (self.snack.act == snack_actions['L']).all():\n",
    "                self.snack.act = snack_actions['L']\n",
    "        if act ==2 :\n",
    "            if not (self.snack.act == snack_actions['U']).all():\n",
    "                self.snack.act = snack_actions['U']\n",
    "        if act ==3 :        \n",
    "            if not (self.snack.act == snack_actions['D']).all():\n",
    "                self.snack.act = snack_actions['D']\n",
    "        #移动\n",
    "        self.snack.move()\n",
    "        self.time_after_last += 1\n",
    "        #迭代一百次后停止\n",
    "        if self.time_after_last == 100:\n",
    "            self.gameover = True\n",
    "            reward = self.dead\n",
    "            self.time_after_last = 0\n",
    "            Done = True\n",
    "        #吃到苹果\n",
    "        if (self.snack.position == self.apple.position).all():\n",
    "            self.apple.eaten()\n",
    "            self.snack.snack_len += 1\n",
    "            self.time_after_last = 0\n",
    "            Done = True\n",
    "        #死亡\n",
    "        elif self.snack.deadcheck(self.snack.position):\n",
    "            self.gameover = True\n",
    "            reward = self.dead\n",
    "            self.time_after_last = 0\n",
    "            Done = True\n",
    "        \n",
    "        return reward, Done, len(self.snack)"
   ],
   "id": "3c0e88b7eec6a9b2",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "#DQN用到的4层MLP（多层感知机，ANN人工神经网络）\n",
    "## hidden_layer_size = 20\n",
    "#隐藏层的大小\n",
    "\n",
    "class QNetwork(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        super(QNetwork, self).__init__()\n",
    "        self.fc1 = nn.Linear(input_dim, hidden_dim)\n",
    "        self.fc2 = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.fc3 = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.fc4 = nn.Linear(hidden_dim, output_dim)\n",
    "        self.relu = nn.ReLU()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        l1 = self.relu(self.fc1(x.float()))\n",
    "        l2 = self.relu(self.fc2(l1))\n",
    "        l3 = self.relu(self.fc3(l2))\n",
    "        l4 = self.fc4(l3)\n",
    "        return l4\n",
    "        \n",
    "def get_network_input(player, apple):\n",
    "    proximity = player.getproximity()\n",
    "    x = torch.cat([torch.from_numpy(player.pos).double(), \n",
    "                   torch.from_numpy(apple.pos).double(), \n",
    "                   torch.from_numpy(player.dir).double(), torch.tensor(proximity).double()])\n",
    "    return x"
   ],
   "id": "c35a23bf9128653"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-15T04:58:35.496860Z",
     "start_time": "2024-11-15T04:58:35.484633Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#存储历史经验数据\n",
    "class ReplayMemory(object):\n",
    "    def __init__(self, max_size):\n",
    "        self.max_size = max_size\n",
    "        self.buffer = []\n",
    "        \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        experience = (state, action, reward, next_state, done)\n",
    "        self.buffer.append(experience)\n",
    "        \n",
    "    def sample(self, batch_size):\n",
    "        state_batch = []\n",
    "        action_batch = []\n",
    "        reward_batch = []\n",
    "        next_state_batch = []\n",
    "        done_batch = []\n",
    "        \n",
    "        batch = random.sample(self.buffer, batch_size)\n",
    "        \n",
    "        for experience in batch:\n",
    "            state, action, reward, next_state, done = experience\n",
    "            state_batch.append(state)\n",
    "            action_batch.append(action)\n",
    "            reward_batch.append(reward)\n",
    "            next_state_batch.append(next_state)\n",
    "            done_batch.append(done)\n",
    "        \n",
    "        return (state_batch, action_batch, reward_batch, next_state_batch, done_batch)\n",
    "    \n",
    "    def truncate(self):\n",
    "        self.buffer = self.buffer[-self.max_size:]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ],
   "id": "319a4e6955b158fc",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-16T01:34:49.162519Z",
     "start_time": "2024-11-16T01:34:44.345110Z"
    }
   },
   "cell_type": "code",
   "source": "\n",
   "id": "9701606f3d5b43fe",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.6.1 (SDL 2.28.4, Python 3.12.7)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001B[1;31mSystemExit\u001B[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\program\\python3\\Lib\\site-packages\\IPython\\core\\interactiveshell.py:3585: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "execution_count": 1
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
